package com.sena.crawler.thread;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.sena.crawler.SenaCrawler;
import com.sena.crawler.domian.DeathLink;
import com.sena.crawler.domian.Link;
import com.sena.crawler.domian.LinkCount;
import com.sena.crawler.enums.*;
import com.sena.crawler.exception.SenaCrawlerException;
import com.sena.crawler.model.PageRequest;
import com.sena.crawler.rundata.*;
import com.sena.crawler.util.*;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.net.Proxy;
import java.util.*;
import java.util.concurrent.TimeUnit;

/**
 * crawler thread
 *
 * @author seven-10-10 10:58:19
 */
public class CrawlerThread implements Runnable {
    private static Logger logger = LoggerFactory.getLogger(CrawlerThread.class);

    private SenaCrawler crawler;
    private boolean running;
    private volatile boolean toStop;
    private JedisUtilNew jedisUtil;

    public CrawlerThread(SenaCrawler crawler) {
        this.crawler = crawler;
        this.running = true;
        this.toStop = false;
        this.jedisUtil = crawler.getRunConf().getJedisUtil();
    }

    public void toStop() {
        this.toStop = true;
    }

    public boolean isRunning() {
        return running;
    }

    @Override
    public void run() {
        System.out.println(">>>>>>>>>>>进入线程");
        while (!toStop) {
            try {
                // ------- url ----------
                running = false;
                crawler.tryFinish();
                Link ln = crawler.getRunData().getUrl();
                String link = ln.getLink();
                running = true;
                logger.info(">>>>>>>>>>> sena crawler, process link : {}", link);
                if (!UrlUtil.isUrl(link)) {
                    continue;
                }
                for (int i = 0; i < (1 + crawler.getRunConf().getFailRetryCount()); i++) {
                    boolean ret = false;
                    try {
                        // make request
                        PageRequest pageRequest = makePageRequest(link);
                        // parse
                        if (ln.getType().equals("paging")) {
                            ret = processPagingPages(ln);
                        } else {
                            ret = processPage(pageRequest, ln);
                        }
                    } catch (Throwable e) {
                        logger.info(">>>>>>>>>>> sena crawler proocess error.", e);
                    }

                    if (crawler.getRunConf().getPauseMillis() > 0) {
                        try {
                            TimeUnit.MILLISECONDS.sleep(crawler.getRunConf().getPauseMillis());
                        } catch (InterruptedException e) {
                            logger.info(">>>>>>>>>>> sena crawler thread is interrupted. 2{}", e.getMessage());
                        }
                    }
                    if (ret) {
                        break;
                    }
                }
                jedisUtil.delSetValue(LinkTypeEnums.UN_VISITED_URL.getCode(),ln.getLink());
                jedisUtil.addSetValue(LinkTypeEnums.VISITED_URL.getCode(),ln.getLink());
            } catch (Throwable e) {
                if (e instanceof InterruptedException) {
                    logger.info(">>>>>>>>>>>线程被打断", e.getMessage());
                } else if (e instanceof SenaCrawlerException) {
                    logger.info(">>>>>>>>>>>线程被打断", e.getMessage());
                    logger.info(">>>>>>>>>>> sena crawler thread {}", e.getMessage());
                } else {
                    logger.error(e.getMessage(), e);
                }
            }

        }
        logger.info(">>>>>>>>>>>出线程");
    }

    /**
     * make page request
     *
     * @param link
     * @return PageRequest
     */
    private PageRequest makePageRequest(String link) {
        String userAgent = crawler.getRunConf().getUserAgentList().size() > 1
                ? crawler.getRunConf().getUserAgentList().get(new Random().nextInt(crawler.getRunConf().getUserAgentList().size()))
                : crawler.getRunConf().getUserAgentList().size() == 1 ? crawler.getRunConf().getUserAgentList().get(0) : null;
        Proxy proxy = null;
        if (crawler.getRunConf().getProxyMaker() != null) {
            proxy = crawler.getRunConf().getProxyMaker().make();
        }

        PageRequest pageRequest = new PageRequest();
        pageRequest.setUrl(link);
        pageRequest.setParamMap(crawler.getRunConf().getParamMap());
        pageRequest.setCookieMap(crawler.getRunConf().getCookieMap());
        pageRequest.setHeaderMap(crawler.getRunConf().getHeaderMap());
        pageRequest.setUserAgent(userAgent);
        pageRequest.setReferrer(crawler.getRunConf().getReferrer());
        pageRequest.setIfPost(crawler.getRunConf().isIfPost());
        pageRequest.setTimeoutMillis(crawler.getRunConf().getTimeoutMillis());
        pageRequest.setProxy(proxy);
        pageRequest.setValidateTLSCertificates(crawler.getRunConf().isValidateTLSCertificates());

        return pageRequest;
    }

    /**
     * 处理全部的y分页Url
     *
     * @param
     * @param
     * @return
     */
    private boolean processPagingPages(Link link) {
        String total = JsoupUtil.loadPageSource(makePageRequest(link.getTotalUrl()));
        if (StringUtils.isNotBlank(total) && RegexUtil.isInteger(total)) {
            int sum = Integer.parseInt(total) / 15 == 0 ? 1 : (Integer.parseInt(total) / 15) + 1;
            for (int i = 1; i <= sum; i++) {
                String url = link.getLink().replace("##", String.valueOf(i));
                String pageSource = JsoupUtil.loadPageSource(makePageRequest(url));
                if (pageSource != null) {
                    JSONObject jsonObject = JSONObject.parseObject(pageSource);
                    JSONArray list = (JSONArray) jsonObject.get("data");
                    if (list != null && list.size() > 0) {
                        for (int j = 0; j < list.size(); j++) {
                            JSONObject e = (JSONObject) list.get(j);
                            Link ln = null;
                            if (StringUtils.isNotBlank(e.get("content_source_url").toString())) {
                                ln = new Link((String) e.get("content_source_url"), null, "common", link.getpLink());
                                OtherUrlData otherUrlData = new OtherUrlData();
                                otherUrlData.setScanUrl(ln.getLink());
                                HashSet<String> newUrl = new HashSet<>();
                                newUrl.add(ln.getLink());
                                otherUrlData.setCntWlinks(newUrl);
                                otherUrlData.setScanTime(new Date());
                                otherUrlData.setCntWlink(1);
                                jedisUtil.setMapValue(UrlTypeEnums.URL_OTHER.getCode(), ln.getLink(), otherUrlData);

                            } else {
                                if (StringUtils.isNotBlank(e.get("notice_id").toString())) {
                                    ln = new Link(link.getDetailUrl().replace("##", e.get("notice_id").toString()), null, "common", link.getpLink());
                                    UrlData u = new UrlData(ln.getLink(), e.get("notice_name").toString(), "新发现Url", ln.getLink(), UrlTypeEnums.URL.getCode(), new Date());
                                    jedisUtil.setMapValue(UrlTypeEnums.URL.getCode(), ln.getLink(), u);
                                }
                            }
                            if (ln != null) {
                                crawler.getRunData().addUrl(ln);
                            }
                        }
                    }

                }
            }
        } else {
            logger.info(">>>>>>>>>>> sena crawler thread is interrupted. 2{}");
        }
        if (total == null) {
            return false;
        }
        return true;
    }

    private boolean processPagingPage(PageRequest totalReq, String pagingUrl) {
        String total = JsoupUtil.loadPageSource(totalReq);
        if (StringUtils.isNotBlank(total) && RegexUtil.isInteger(total)) {
            int sum = Integer.parseInt(total) / 15 == 0 ? 1 : (Integer.parseInt(total) / 15) + 1;
            for (int i = 1; i <= sum; i++) {
                String url = pagingUrl.replace("##", String.valueOf(i));
                Link link = new Link(url, null, "common", url);
                crawler.getRunData().addUrl(link);
            }
        } else {
            logger.info(">>>>>>>>>>> sena crawler thread is interrupted. 2{}");
        }
        if (total == null) {
            return false;
        }
        return true;
    }


    /**
     * process page
     *
     * @param pageRequest
     * @return boolean
     */
    private boolean processPage(PageRequest pageRequest, Link ln) throws IllegalAccessException, InstantiationException {
        Document html = crawler.getRunConf().getPageLoader().load(pageRequest);

        //正则表达式处理关键字
        String str = pageRequest.getUrl().substring(pageRequest.getUrl().lastIndexOf("."));
        if (html == null) {
            return false;
        }
        html.setBaseUri(pageRequest.getUrl());
        //处理是是否是死链接
        if (crawler.getRunConf().getSenaCrawlerType().equals(LinkTypeEnums.DIE_LINK.getCode()) && !UrlUtil.checkUrl(ln.getLink())) {
            if (FileTypeEnums.getByCodeType(str) == null) {
                DeathLink deathLink = null;
                if (jedisUtil.hExists(LinkTypeEnums.DIE_LINK.getCode(), ln.getpLink())) {
                    deathLink = (DeathLink) jedisUtil.getMapValue(LinkTypeEnums.DIE_LINK.getCode(), ln.getLink(), DeathLink.class);
                    if (!deathLink.getSlinks().contains(ln.getLink())) {
                        Set<String> set = deathLink.getSlinks();
                        set.add(ln.getLink());
                        deathLink.setScan_time(new Date());
                        deathLink.setCnt_slink(set.size());
                    }
                } else {
                    Set<String> set = new HashSet<>();
                    set.add(ln.getLink());
                    deathLink = new DeathLink(new Date(), ln.getpLink(), set.size(), set);
                }
                jedisUtil.setMapValue(LinkTypeEnums.DIE_LINK.getCode(), ln.getpLink(), JSON.toJSONString(deathLink));
            }
        }

        // limit child spread
        if (crawler.getRunConf().isAllowSpread()) {
            //获取全部
            Set<Element> links = JsoupUtil.findLinks(html, crawler.getRunConf(), crawler);
            if (links != null && links.size() > 0) {
                for (Element item : links) {
                    String url = item.attr("abs:href");
                    if (crawler.getRunConf().validWhiteUrl(url) && !crawler.getRunConf().validBlackUrl(url)) {

                        Link link = new Link(url, LinkTypeEnums.COMMON_LINK.getCode(), html.baseUri(), item.text());
                        //处理后缀
                        dealSuffix(link, crawler);
                        //判断是否需要最大数量的url
                        if (crawler.getRunConf().isNeedMax()) {
                            //判断连接是否在需要访问的url中
                            if (crawler.getRunData().isFind(link)) {
                                boolean flag = dealCount(link, crawler);
                                //判断是大于最大的连接数了
                                if (!flag) {

                                    if (crawler.getRunData().addUrl(link)) {
                                        jedisUtil.addSetValue(LinkTypeEnums.UN_VISITED_URL.getCode(),link.getLink());
                                        if (LinkTypeEnums.COMMON_LINK.getCode().equals(crawler.getRunConf().getSenaCrawlerType())) {
                                            jedisUtil.setSetValue(LinkTypeEnums.COMMON_LINK.getCode(), JSON.toJSONString(link));
                                        }
                                    }
                                }
                            }
                        } else {
                            if (crawler.getRunData().addUrl(link)) {
                                jedisUtil.addSetValue(LinkTypeEnums.UN_VISITED_URL.getCode(),link.getLink());
                                if (LinkTypeEnums.COMMON_LINK.getCode().equals(crawler.getRunConf().getSenaCrawlerType())) {
                                    jedisUtil.setSetValue(LinkTypeEnums.COMMON_LINK.getCode(), JSON.toJSONString(link));
                                }
                            }
                        }

                    }
                }
            }
        }
        /**
         * 是否匹配字符串
         */
        if (crawler.getRunConf().isAllowRegex()) {
            //
            matchRegex(html, crawler);
        }

        /**
         * 执行匹配g关键字
         */
        if (crawler.getRunConf().getKeyword() != null && crawler.getRunConf().getKeyword().size() > 0) {
            matchKeyWorld(html, crawler);
        }


        // ------- pagevo ----------
        // limit unvalid-page parse, only allow spread child, finish here
        if (!crawler.getRunConf().validWhiteUrl(pageRequest.getUrl()) && crawler.getRunConf().validBlackUrl(pageRequest.getUrl())) {
            return true;
        }

        return true;
    }

    /**
     * 保存执行的存在的url 和他的次数
     *
     * @param link
     * @param crawler
     */
    private boolean dealCount(Link link, SenaCrawler crawler) {
        if (!link.getLink().equals(crawler.getRunConf().getRootLink())) {
            String url = link.getLink();
            String rel = null;
            if (url.contains("?")) {
                rel = url.substring(0, url.lastIndexOf("?"));
            } else {
                rel = url;
            }
            LinkCount linkCount = null;
            if (jedisUtil.hExists(LinkTypeEnums.COUNT_LINK.getCode(), rel)) {
                linkCount = (LinkCount) jedisUtil.getMapValue(LinkTypeEnums.COUNT_LINK.getCode(), rel, LinkCount.class);
                if (linkCount.getCount() >= crawler.getRunConf().getMaxUrl()) {
                    return true;
                } else {
                    linkCount.setCount(linkCount.getCount() + 1);
                }
            } else {
                linkCount = new LinkCount();
                linkCount.setUrl(rel);
                linkCount.setCount(1);
            }
            jedisUtil.setMapValue(LinkTypeEnums.COUNT_LINK.getCode(), rel, JSON.toJSONString(linkCount));

            return false;
        } else {
            return false;
        }
    }


    /**
     * 匹配关键字 ,处理正则表达式
     *
     * @param html
     * @param crawler
     */
    private void matchRegex(Document html, SenaCrawler crawler) {
        if (crawler.getRunConf().isAllowRegex()) {
            //处理正则
            HashMap<String, HashMap<String, Set<String>>> mapHashMap = RegexUtil.regex(html.text(), html.baseUri());
            if (!mapHashMap.isEmpty()) {
                RegexData regexData = new RegexData();
                regexData.setScanDate(new Date());
                regexData.setListHashMap(mapHashMap);
                regexData.setUrl(html.baseUri());
                jedisUtil.setMapValue(LinkTypeEnums.REGEX_DATA.getCode(), html.baseUri(), regexData);
            }
        }
    }

    /**
     * 匹配关键字
     *
     * @param html
     * @param crawler
     */
    private void matchKeyWorld(Document html, SenaCrawler crawler) {
        if (crawler.getRunConf().getKeyword().size() > 0) {
            //处理匹配
            Set<String> strings = crawler.getRunConf().getKeyword();
            KeyWorldData keyWorldData = new KeyWorldData();
            List<String> kList = new ArrayList<>();
            strings.forEach(e -> {
                if (html.toString().contains(e)) {
                    kList.add(e);
                }
            });
            if (kList.size() > 0) {
                keyWorldData.setUrl(html.baseUri());
                keyWorldData.setkList(kList);
                keyWorldData.setDate(new Date());
                jedisUtil.setMapValue(LinkTypeEnums.KEY_WORLD.getCode(), html.baseUri(), keyWorldData);
            }
        }
    }


    /**
     * 处理后缀
     *
     * @param link
     * @param crawler
     */
    private void dealSuffix(Link link, SenaCrawler crawler) {
        if (crawler.getRunConf().getSuffixs().size() > 0) {
            HashSet<String> strings = (HashSet<String>) crawler.getRunConf().getSuffixs();
            strings.forEach(e -> {
                if (link.getLink().contains(e)) {
                    if (link.getLink().contains("?" + e) || link.getLink().contains("&" + e)) {
                        String str = link.getLink().substring(0, link.getLink().lastIndexOf(e) - 1);
                        link.setLink(str);
                    } else {
                        String str = link.getLink().substring(0, link.getLink().lastIndexOf(e));
                        link.setLink(str);
                    }
                }
            });
        }
    }


}